package linearRegression.featureExtractor;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Scanner;

public class LinearRegression_tvt {

//	public static String movieFile = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/movie_titles.txt";
//	public static String trainingSetPath = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/training_set/";
	public static String trainingSetPath = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/training_set_new/";
	public static String testFile = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/qualifying.txt";
	public static String testAnsFile = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/judging.txt";
	public static String validationPath = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/probe_combine.txt";
	public static String validationOutput = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/validation_out.txt";
	public static String outputTrain = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/train_out_linear.txt";
	public static String outputTest = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/test_out_linear.txt";
	public static String outputValid = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/valid_out_linear.txt";
//	public static String predictFile = "/Users/macpro724/Desktop/ml_prj/data/netflix/download/linearResult.txt";
	
	public final int MAX_RATINGS = 99072113;
	public final int MAX_CUSTOMERS = 480190;
	public final int MAX_MOVIES = 17771;
	
	public final int VAL_RATINGS = 1408396;
	
	public int[] custId;
	public short[] movieId;
	public float[] rating;
	public float[] y;
	
	public float[] movieBias;
	public float[] custBias;
	public int[] movieCount;
	public int[] custCount;
	
	public float overmean;
	public int totalScore;
	public int index;
	public int userIndex;
	public int vindex;
	
	public Map<Integer, Integer> custMap;
	public Map<Integer, Integer> custMapR;
	
	public double w_left;
	public double w_movie_bias;
	public double w_user_bias;
	
	public int[] custId_v;
	public short[] movieId_v;
	public float[] rating_v;
	
	public LinearRegression_tvt(){
		custId = new int[MAX_RATINGS];
		movieId = new short[MAX_RATINGS];
		rating = new float[MAX_RATINGS];
		y = new float[MAX_RATINGS];
		
		custId_v = new int[VAL_RATINGS];
		movieId_v = new short[VAL_RATINGS];
		rating_v = new float[VAL_RATINGS];
		
		movieBias = new float[MAX_MOVIES];
		custBias = new float[MAX_CUSTOMERS];
		movieCount = new int[MAX_MOVIES];
		custCount = new int[MAX_CUSTOMERS];
		custMap = new HashMap<Integer, Integer>();
		custMapR = new HashMap<Integer, Integer>();
		
		w_left = 1.0;
		w_movie_bias = 1.0;
		w_user_bias = 1.0;
		
		for (int i = 0; i < MAX_MOVIES; i ++) movieBias[i] = 0;
		for (int i = 0; i < MAX_CUSTOMERS; i ++) custBias[i] = 0;
		for (int i = 0; i < MAX_MOVIES; i ++) movieCount[i] = 0;
		for (int i = 0; i < MAX_CUSTOMERS; i ++) custCount[i] = 0;
	}
	
	public void loadData(){
		File dir = new File(LinearRegression_tvt.trainingSetPath);
		File[] files = dir.listFiles();
		index = 0;
		userIndex = 0;
		for (File file : files){
			
			System.out.println(file.getName() +"\t" + index);
			try{
				BufferedReader br = new BufferedReader(new FileReader(file));
				
				String line;
				short movie = 0;
				int cust = 0;
				while ((line = br.readLine()) != null){
					if (line.endsWith(":"))
					{
						movie = Short.parseShort(line.substring(0, line.length()-1));
					}else{
						String[] s = line.split(",");
						cust = Integer.parseInt(s[0]);
						if (custMap.containsKey(cust)){
							cust = custMap.get(cust);
						}else{
							custMap.put(cust, userIndex);
							custMapR.put(userIndex, cust);
							
							cust = userIndex;							
							userIndex ++;
						}
						custId[index] = cust;
						movieId[index] = (short)movie;
						rating[index] = Float.parseFloat(s[1]);
						
						totalScore = (int) (totalScore + rating[index]);
						
//						custBias[custId[index]] += rating[index];
						custCount[custId[index]] ++;
						movieCount[movieId[index]]++;
						movieBias[movieId[index]] += rating[index];
						
						index = index + 1;
					}
				}
				
				
				br.close();
			}catch(Exception e){
				e.printStackTrace();
			}
		}
		
		System.out.println("Total Number\t" + index);
		System.out.println("Total User\t" + userIndex);
	}
	
	public void prpData(){
		float p1 = 25.0f;
		float p2 = 10.0f;
		//calc overmean
		overmean = 0;
		for (int i = 0; i < MAX_MOVIES; i ++) 
		{
			overmean += movieBias[i];
		}
		overmean = (float)overmean/(float)index;
		
		//calc movieBias
		/*
		for (int i = 0; i < MAX_MOVIES; i ++) 
		{
			movieBias[i] = (float)(movieBias[i] + 10*3.6f)/((float)movieCount[i] + 10.0f) - overmean;
			System.out.println(movieBias[i]);
		}
		*/
		for (int i = 0; i < MAX_MOVIES; i++) movieBias[i] = 0;
		for (int i = 0; i < index; i++){
			int movie = movieId[i];
			movieBias[movie] += rating[i] - overmean;
		}
		for (int i = 0; i < MAX_MOVIES; i++){
			movieBias[i] = movieBias[i]/((float)movieCount[i] + p1);
		}

		//calc custBias
		for (int i = 0; i < MAX_CUSTOMERS; i ++) custBias[i] = 0;
		
		for (int i = 0; i < index; i++){
			int cust = custId[i];
			int movie = movieId[i];
			custBias[cust] += rating[i] - overmean - movieBias[movie];
		}
		
		for (int i = 0; i < MAX_CUSTOMERS; i ++)
		{
			custBias[i] = custBias[i]/((float)custCount[i] + p2);
		}
		
		
		//calc y
		for (int i = 0; i < MAX_RATINGS; i ++){
			y[i] = rating[i] - overmean;
		}
	}
	
	public void calcW() throws IOException{
		double lRate = 0.0001;
		double K = 0.0015;
		int minEpoch = 3;
		int maxEpoch = 5;
		double meanImprove = 0.0001;

		double rmse = 0;
		double rmseLast = 2;
		double sq = 0;
		
		double tempFunc = 0;
		double error = 0;
		
		double rmse_v = 0;
		double rmse_vlast = 2;
		System.out.println("index\t" + index);
		System.out.println("vindex\t" + vindex);
		/*
		try{
			System.out.println("begin training");
		int flag = System.in.read();
		while(flag > 0){
			w_left = 1;
			w_movie_bias = 1;
			w_user_bias = 1;
			Scanner scanner = new Scanner(System.in);
				lRate = Double.parseDouble(scanner.next());
				K = Double.parseDouble(scanner.next());
				System.out.println("lrate\t"+lRate);
				System.out.println("K\t"+K);
				*/

//		BufferedWriter bw = new BufferedWriter(new FileWriter(LinearRegression_tvt.validationOutput));
		for (int e = 0; (e < minEpoch) || ((e < maxEpoch) && (rmseLast > rmse + meanImprove)) ;e++)
		{
//			bw.write("TrainingBegin!!!\n");
			rmseLast = rmse;
			sq = 0;
			for (int i = 0; i < index; i++){
				tempFunc = w_left + w_user_bias*custBias[custId[i]] + w_movie_bias*movieBias[movieId[i]];
				error = y[i] - tempFunc;
				sq += error*error;
				
				w_left = w_left + lRate*(error - K*w_left);
				w_movie_bias = w_movie_bias + lRate*(error*movieBias[movieId[i]] - K*w_movie_bias);
				w_user_bias = w_user_bias + lRate*(error*custBias[custId[i]] - K*w_user_bias);
				
				
//				if (i%1000000 == 0)
//				{
//					rmse_v = getValidation();
//					System.out.println("" + i + "\t" + rmse_v );
//					bw.write("" + i + "\t" + rmse_v + "\n");
//					if (rmse_v > rmse_vlast + 0.001) break;
//					else{
//						rmse_vlast = rmse_v;
//					}
//				}
				
			}
			rmse = Math.sqrt(sq/index);
			
			System.out.println("Round " + e + "\t" + "RMSE: " + rmse);
//			System.out.println("=====Result======");
//			System.out.println("OverMean\t" + overmean);
//			System.out.println("Const\t" + w_left);
//			System.out.println("W_movie\t" + w_movie_bias);
//			System.out.println("W_user\t" + w_user_bias);
		}
//		bw.close();
/*			testModel();
		}}
		}catch (Exception e){
			e.printStackTrace();
		}
		*/
			
	}
	
	//linear regression
	//y = w_const + overmean + w_movie*movie_bias + w_user*user_bias;
	public void testModel(){
		BufferedReader testBr;
		BufferedReader ansBr;
		BufferedWriter bw;
		
		try{
			testBr = new BufferedReader(new FileReader(LinearRegression_tvt.testFile));
			ansBr = new BufferedReader(new FileReader(LinearRegression_tvt.testAnsFile));
			
			bw = new BufferedWriter(new FileWriter(LinearRegression_tvt.outputTest));
			
			double rmse = 0;
			double sq = 0;
			int count = 0;
			
			
			String testLine = testBr.readLine();
			String ansLine = ansBr.readLine();
			
			int movie = 0;
			int cust = 0;
			double score = 0;
			double ans;
			double err;
			while ((testLine != null) && (ansLine != null)){
				if (testLine.length() == 0) break;
				if (testLine.endsWith(":")){
					movie = Integer.parseInt(testLine.substring(0, testLine.length()-1));
				}else{
					count ++;
					cust = Integer.parseInt(testLine.substring(0, testLine.indexOf(",")));
					cust = custMap.get(cust);
					
					score = overmean + w_left +w_movie_bias*movieBias[movie] + w_user_bias*custBias[cust];
					score = normValue(score);
					
					ans = Double.parseDouble(ansLine.substring(0, ansLine.indexOf(",")));
					
					bw.write("" + custMapR.get(cust) + "," + movie + "," + score + "," + ans + "\n");
					err = score - ans;
					sq += err*err;
				}
				testLine = testBr.readLine();
				ansLine = ansBr.readLine();
			}
			
			rmse = Math.sqrt(sq/count);			
			System.out.println("Final Result RMSE: " + rmse);
			testBr.close();
			ansBr.close();
			bw.close();
		}catch(Exception e){
			e.printStackTrace();
		}
	}
	
	public double normValue(double score){
		double temp = score;
		if (temp > 5) temp = 5;
		else if (temp < 1) temp = 1;
		return temp;
	}
	
	public double getValidation(){
		double rmse = 0;
		double sq = 0;
		double score = 0;
		double err = 0;
		double ans = 0;
		short movie = 0;
		int cust = 0;
		for (int i = 0; i < vindex; i++){
			movie = movieId_v[i];
			cust = custId_v[i];
			
			score = overmean + w_left +w_movie_bias*movieBias[movie] + w_user_bias*custBias[cust];
			score = normValue(score);
			
			ans = rating_v[i];
			err = score - ans;
			sq += err*err;
		}
		
		rmse = Math.sqrt(sq/vindex);
		
		return rmse;
	}
	
	public void prpValidation(){
		
		BufferedReader br;
		try{
			br = new BufferedReader(new FileReader(LinearRegression_tvt.validationPath));
			String line;
			int cust = 0;
			int movie = 0;
			vindex = 0;
			while ((line = br.readLine()) != null){
				if (!line.endsWith(":")){
					String[] s = line.split(",");
					cust = Integer.parseInt(s[0]);
					if (!custMap.containsKey(cust)){
						custMap.put(cust, userIndex);
						cust = userIndex;
						userIndex++;
						System.out.println("New User\t" + userIndex);
					}else cust = custMap.get(cust);
					
					movieId_v[vindex] = (short)movie;
					custId_v[vindex] = cust;
					rating_v[vindex] = Float.parseFloat(s[1]);
					
					vindex ++;
				}else{
					movie = Integer.parseInt(line.substring(0, line.length()-1));
				}
			}
			br.close();
		}catch(Exception e){
			e.printStackTrace();
		}
		
	}
	
	public void outputResult(){
		BufferedWriter bw;
		short movie = 0;
		int cust = 0;
		float rate = 0f;
		float score;
		
		try{
			//training
			bw = new BufferedWriter(new FileWriter(LinearRegression_tvt.outputTrain));
			
			for (int i = 0; i < index; i++){
				movie = movieId[i];
				cust = custId[i];
				rate = rating[i];
				
				score = (float)(overmean + w_left +w_movie_bias*movieBias[movie] + w_user_bias*custBias[cust]);
				score = (float) normValue(score);
				
				bw.write("" + custMapR.get(cust) + "," + movie + "," + score + "," + rate + "\n");
			}
			
			bw.close();
			
			bw = new BufferedWriter(new FileWriter(LinearRegression_tvt.outputValid));
			
			for (int i = 0; i < vindex; i++){
				movie = movieId_v[i];
				cust = custId_v[i];
				rate = rating_v[i];
				
				score = (float)(overmean + w_left +w_movie_bias*movieBias[movie] + w_user_bias*custBias[cust]);
				score = (float) normValue(score);
				
				bw.write("" + custMapR.get(cust) + "," + movie + "," + score + "," + rate + "\n");
			}
			
			bw.close();
		}catch(Exception e){
			
		}
	}
	
	public static void main(String[] args) throws IOException {
		
		LinearRegression_tvt engine = new LinearRegression_tvt();
		System.out.println("after initializing!!");
		engine.loadData();
		System.out.println("after loading!!");
		engine.prpData();
		engine.prpValidation();
		System.out.println("after prepare!!!");
		engine.calcW();
		
		System.out.println("=====Result======");
		System.out.println("OverMean\t" + engine.overmean);
		System.out.println("Const\t" + engine.w_left);
		System.out.println("W_movie\t" + engine.w_movie_bias);
		System.out.println("W_user\t" + engine.w_user_bias);
		
		engine.testModel();
		engine.outputResult();
		

	}

}
